{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据导入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0707 20:19:41.704542 139980311164736 deprecation.py:323] From <ipython-input-2-6feb25f5d59b>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "W0707 20:19:41.705710 139980311164736 deprecation.py:323] From /home/wangxi/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "W0707 20:19:41.708379 139980311164736 deprecation.py:323] From /home/wangxi/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0707 20:19:41.923345 139980311164736 deprecation.py:323] From /home/wangxi/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "W0707 20:19:41.926358 139980311164736 deprecation.py:323] From /home/wangxi/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "W0707 20:19:41.969250 139980311164736 deprecation.py:323] From /home/wangxi/anaconda3/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n",
      "(55000, 10)\n",
      "(5000, 784)\n",
      "(5000, 10)\n",
      "(10000, 784)\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('.',one_hot=True)\n",
    "\n",
    "print(mnist.train.images.shape)\n",
    "print(mnist.train.labels.shape)\n",
    "\n",
    "print(mnist.validation.images.shape)\n",
    "print(mnist.validation.labels.shape)\n",
    "\n",
    "print(mnist.test.images.shape)\n",
    "print(mnist.test.labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义模型的计算图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义学习率\n",
    "learning_rate = tf.placeholder(tf.float32,name='learning_rate')\n",
    "\n",
    "# 定义输入\n",
    "x = tf.placeholder(tf.float32,[None,784],name='x')\n",
    "\n",
    "# 定义权重初始化方法\n",
    "def initial_weight(shape):\n",
    "    return tf.truncated_normal(shape,stddev=4.0 / sum(shape))\n",
    "\n",
    "# 第一隐层500个神经元\n",
    "w1 = tf.Variable(initial_weight([784,500]),name='weight1')\n",
    "# 添加l2正则\n",
    "tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(0.0001)(w1))\n",
    "b1 = tf.Variable(initial_weight([500]),name='b1')\n",
    "logit1 = tf.matmul(x,w1) + b1\n",
    "output1 = tf.nn.relu(logit1)\n",
    "\n",
    "# # 第二层64个神经元\n",
    "# w2 = tf.Variable(initial_weight([512,128]),name='weight2')\n",
    "# # 添加l2正则\n",
    "# tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(0.0001)(w2))\n",
    "# b2 = tf.Variable(initial_weight([128]),name='b2')\n",
    "# logit2 = tf.matmul(output1,w2) + b2\n",
    "# output2 = tf.nn.relu(logit2)\n",
    "\n",
    "# # 第三层32个神经元\n",
    "# w3 = tf.Variable(initial_weight([64,32]),name='weight3')\n",
    "# # 添加l2正则\n",
    "# tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(0.01)(w3))\n",
    "# b3 = tf.Variable(initial_weight([32]),name='b3')\n",
    "# logit3 = tf.matmul(output2,w3) + b3\n",
    "# output3 = tf.nn.relu(logit3)\n",
    "\n",
    "# 第四层输出层\n",
    "w4 = tf.Variable(initial_weight([500,10]),name='weight4')\n",
    "tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(0.0001)(w4))\n",
    "b4 = tf.Variable(initial_weight([10]),name='b4')\n",
    "logits = tf.matmul(output1,w4) + b4\n",
    "\n",
    "# 定义groudtruth\n",
    "y = tf.placeholder(tf.float32,[None,10],name='y')\n",
    "\n",
    "# 定义经过softmax的交叉熵损失均值\n",
    "cross_entropy_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y,logits=logits))\n",
    "tf.add_to_collection('losses',cross_entropy_loss)\n",
    "cross_entropy_loss = tf.add_n(tf.get_collection('losses'))\n",
    "# 定义优化步骤\n",
    "step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy_loss)\n",
    "\n",
    "# 定义准确率\n",
    "corret_prediction = tf.equal(tf.argmax(y,1),tf.argmax(logits,1))\n",
    "# 计算准确率，mean求一批训练的平均值，correctprediction中存的是bool值，cast转换成浮点数\n",
    "accuracy = tf.reduce_mean(tf.cast(corret_prediction,tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199 0.953125 0.9378 0.792\n",
      "399 0.984375 0.9632 0.78408\n",
      "599 0.984375 0.9686 0.7762392\n",
      "799 1.0 0.9726 0.768476808\n",
      "999 1.0 0.974 0.76079203992\n",
      "1199 1.0 0.9778 0.7531841195208\n",
      "1399 1.0 0.9786 0.745652278325592\n",
      "1599 1.0 0.9782 0.7381957555423361\n",
      "1799 1.0 0.98 0.7308137979869127\n",
      "1999 0.9921875 0.9814 0.7235056600070435\n",
      "2199 1.0 0.9806 0.7162706034069731\n",
      "2399 1.0 0.9826 0.7091078973729034\n",
      "2599 1.0 0.9822 0.7020168183991743\n",
      "2799 1.0 0.9818 0.6949966502151825\n",
      "2999 1.0 0.9822 0.6880466837130307\n",
      "3199 1.0 0.9826 0.6811662168759004\n",
      "3399 1.0 0.9834 0.6743545547071413\n",
      "3599 1.0 0.9834 0.6676110091600699\n",
      "3799 1.0 0.9836 0.6609348990684691\n",
      "3999 1.0 0.9824 0.6543255500777845\n",
      "4199 1.0 0.9824 0.634695783575451\n",
      "4399 1.0 0.9838 0.6156549100681874\n",
      "4599 1.0 0.9834 0.5971852627661418\n",
      "4799 1.0 0.9848 0.5792697048831575\n",
      "4999 1.0 0.9846 0.5618916137366629\n",
      "5199 1.0 0.9832 0.5450348653245629\n",
      "5399 1.0 0.984 0.528683819364826\n",
      "5599 1.0 0.9844 0.5128233047838812\n",
      "5799 1.0 0.985 0.49743860564036474\n",
      "5999 1.0 0.9854 0.48251544747115377\n",
      "6199 1.0 0.9852 0.46803998404701913\n",
      "6399 1.0 0.9852 0.45399878452560855\n",
      "6599 1.0 0.9846 0.44037882098984027\n",
      "6799 1.0 0.9858 0.427167456360145\n",
      "6999 1.0 0.985 0.4143524326693407\n",
      "7199 1.0 0.9852 0.4019218596892605\n",
      "7399 1.0 0.9862 0.3898642038985827\n",
      "7599 1.0 0.9854 0.3781682777816252\n",
      "7799 1.0 0.986 0.3668232294481764\n",
      "7999 1.0 0.986 0.3558185325647311\n",
      "8199 1.0 0.9858 0.34514397658778917\n",
      "8399 1.0 0.9854 0.3347896572901555\n",
      "8599 1.0 0.986 0.3247459675714508\n",
      "8799 1.0 0.985 0.31500358854430727\n",
      "8999 1.0 0.9856 0.305553480887978\n",
      "9199 1.0 0.9854 0.29638687646133866\n",
      "9399 1.0 0.9858 0.2874952701674985\n",
      "9599 1.0 0.9852 0.27887041206247354\n",
      "9799 1.0 0.9846 0.2705042997005993\n",
      "9999 1.0 0.9864 0.26238917070958134\n",
      "10199 1.0 0.985 0.2545174955882939\n",
      "10399 1.0 0.9858 0.24688197072064508\n",
      "10599 1.0 0.9868 0.23947551159902572\n",
      "10799 1.0 0.9864 0.23229124625105493\n",
      "10999 1.0 0.9868 0.22532250886352329\n",
      "11199 1.0 0.9864 0.21856283359761758\n",
      "11399 1.0 0.9858 0.21200594858968905\n",
      "11599 1.0 0.9864 0.20564577013199836\n",
      "11799 1.0 0.9868 0.1994763970280384\n",
      "11999 1.0 0.986 0.19349210511719725\n",
      "0.9866\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6sAAAFpCAYAAAB6Xb4TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xl4VOXd//HPnT0kIQsESAhhSRRBUcQA7riCtnUDF7CiPtUfVm21Vlu356mtivpY20fb2lZttWJbqFpc6haEoqJ1IULCqpgEYUKYEAhJgCRku39/TDImEMh2JrPk/bquXCcz58w53yGBaz5873PfxlorAAAAAAACSZi/CwAAAAAA4ECEVQAAAABAwCGsAgAAAAACDmEVAAAAABBwCKsAAAAAgIBDWAUAAAAABBzCKgAAAAAg4BBWAQAAAAABh7AKAAAAAAg4hFUAAAAAQMCJ8HcBBxo8eLAdNWqUv8sAAAAAAPjA559/vtNam9rZcQEXVkeNGqW8vDx/lwEAAAAA8AFjzJauHMcwYAAAAABAwCGsAgAAAAACDmEVAAAAABBwCKsAAAAAgIBDWAUAAAAABBzCKgAAAAAg4BBWAQAAAAABh7AKAAAAAAg4hFUAAAAAQMDpNKwaY541xuwwxqw7xH5jjPmNMabQGLPGGDOpzb5rjDFftXxd42ThAAAAAIDQ1ZXO6l8knXeY/edLOqLla56kP0iSMSZF0n2SpkqaIuk+Y0xyb4oFAAAAAPQPEZ0dYK39wBgz6jCHXCRpgbXWSvrEGJNkjEmTdIakd621FZJkjHlXntC7sLdFoxN1ddKKFVJjo78rCSxHHSWNHt031/rkE2n37r65FgAAANDW+PHSyJH+rqLXOg2rXTBckqvN45KW5w71/EGMMfPk6coqMzPTgZL6uSeflO64w99VBJ7MTKm4WAoP9+11PvtMOukk314DAAAAOJTf/U66+WZ/V9FrToRV08Fz9jDPH/yktU9LelqScnJyOjwG3bBypZSRIb38sr8rCRwffSTdfru0fLl0zjm+vdZf/iLFxkq5uVJUlG+vBQAAABxo1Ch/V+AIJ8JqiaQRbR5nSCptef6MA55/z4HroTP5+dIJJ0hTp/q7ksBx3HHS/fdLzz/v27C6f7+0aJF0ySXSaaf57joAAABAiHNi6ZrXJV3dMivwiZKqrLXbJeVKmm6MSW6ZWGl6y3PwpZoa6auvPOEM34iJka64Qlq8WNqzx3fXeeMNz72qV1/tu2sAAAAA/UBXlq5ZKOljSWONMSXGmOuMMd83xny/5ZC3JBVLKpT0jKSbJKllYqUHJK1s+bq/dbIl+NC6dVJzM2G1I9dc4wnz//yn767x/PNSWprvhxoDAAAAIa4rswHP6WS/ldTh3bvW2mclPduz0tAjBQWe7cSJ/q0jEJ10kpSdLS1YIF17rfPn37FDevtt6bbbfD+JEwAAABDinBgGjECSny8lJITMTdWOMsYzPHf5cmnLFufPv3ChZ7mga65x/twAAABAP0NYDTUFBdKxx0ph/Gg7NHeuZ/vXvzp/7uef90xsdfTRzp8bAAAA6GdINKGkuVlas4YhwIczapQ0bZonWFoHV0lau1ZavZqJlQAAAACHEFZDyddfe2a6ZXKlw7vmGs+MyZ9+6tw5FyyQIiKkOYe9xRsAAABAFxFWQ0l+vmdLWD28WbOk2FhPd9UJjY2eYcXf+paUmurMOQEAAIB+jrAaSgoKPPeqHnOMvysJbAMHSjNnSosWSXV1vT/f0qWS283ESgAAAICDCKuhpKBAOvJIacAAf1cS+K6+WqqslN54o/fnWrBASkmRvv3t3p8LAAAAgCTCamjJz2cIcFedfbaUnt77ocBVVdIrr0izZ0vR0c7UBgAAAICwGjIqKz1rhxJWuyY8XLrqKuntt6UdO3p+npde8gwlZggwAAAA4CjCaqhYs8azZdmarrv6aqmpSfr733t+jgULpLFjpcmTnasLAAAAAGE1ZBQUeLZ0Vrvu6KOlE07wBM6eKC6WVqzwdFWNcbY2AAAAoJ8jrIaK/Hxp8GApLc3flQSXa66RVq+W1q7t/mtfeMETUq+6yvm6AAAAgH6OsBoqCgo8XVU6fN0ze7YUEdH97qq1ntecdZY0YoRvagMAAAD6McJqKGhslNat437VnkhN9Sw589e/ev4cu+qjjzzDgJlYCQAAAPAJwmoo2LRJ2r+f+1V76pprJLdbWrq06695/nkpLk665BLf1QUAAAD0Y4TVUJCf79kSVnvmW9+SUlK6vuZqba304ovSpZdK8fG+rQ0AAADopwiroaCgQIqKko46yt+VBKfoaGnOHOnVV6Wqqs6Pf+01qbras/QNAAAAAJ8grIaCggJp/HhPYEXPXH21VFcnvfRS58c+/7xnUqUzzvB5WQAAAEB/RVgNBfn5DAHurcmTPZ3pzoYCb98uLVkizZ0rhfHXBwAAAPAVPm0Hu7IyzxdhtXeM8XRXP/xQKio69HF/+5vU3MwQYAAAAMDHCKvBrqDAs2XZmt676ipPaH3hhY73W+vpvJ54ojR2bN/WBgAAAPQzhNVg1xpW6az23ogR0llnSQsWeILpgfLzPevZ0lUFAAAAfI6wGuzy86WMDM/SK+i9a66RNm/2DAc+0PPPeyaxuuKKvq8LAAAA6GcIq8GuoIAhwE6aOVOKi/N0V9tqaJD+/nfpggv4jwEAAACgDxBWg1ldnfTFFwwBdlJcnHTppdKLL0q1td88/847Unm5p/MKAAAAwOcIq8FswwapqYmw6rRrrpGqq6VXX/3muQULpNRU6bzz/FcXAAAA0I8QVoNZfr5nS1h11rRpUmbmN0OBKyqk11+XrrxSioz0b20AAABAP0FYDWYFBZ5hq1lZ/q4ktISFSXPnSkuWSKWl0j/+IdXXMwQYAAAA6EOE1WBWUCBNmCCFh/u7ktAzd67U3OyZVGnBAumYY5jICgAAAOhDhNVgZa0nrDIE2DfGjpVOPFH6v/+TPvnE01U1xt9VAQAAAP0GYTVYbd0qVVbS7fOlq6/2DAMOC5O++11/VwMAAAD0K4TVYFVQ4NnSWfWdK66QoqKk6dOltDR/VwMAAAD0KxH+LgA9VFDgGZY6YYK/KwldKSnS229Lo0b5uxIAAACg3yGsBqv8fM8swPHx/q4ktJ11lr8rAAAAAPolhgEHq4IC7lcFAAAAELIIq8Fozx6pqIj7VQEAAACELMJqMFq71rMlrAIAAAAIUYTVYJSf79kyDBgAAABAiOpSWDXGnGeM+dIYU2iMuauD/SONMcuMMWuMMe8ZYzLa7PtfY8y6lq8rnCy+3yookJKTpYyMzo8FAAAAgCDUaVg1xoRLelLS+ZLGS5pjjBl/wGGPSVpgrT1W0v2SHm557bclTZI0UdJUST8xxgx0rvx+qqDAMwTYGH9XAgAAAAA+0ZXO6hRJhdbaYmttvaRFki464Jjxkpa1fL+8zf7xkt631jZaa/dJKpB0Xu/L7seamjz3rHK/KgAAAIAQ1pWwOlySq83jkpbn2iqQNKvl+0skJRhjBrU8f74xZoAxZrCkMyWN6F3J/VxhoVRTw/2qAAAAAEJaV8JqR2NN7QGP75A0zRizWtI0SdskNVprl0h6S9J/JC2U9LGkxoMuYMw8Y0yeMSavvLy8O/X3PwUFni2dVQAAAAAhrCthtUTtu6EZkkrbHmCtLbXWzrTWHi/p3pbnqlq28621E62158oTfL868ALW2qettTnW2pzU1NQevpV+oqBAioiQxh942zAAAAAAhI6uhNWVko4wxow2xkRJmi3p9bYHGGMGG2Naz3W3pGdbng9vGQ4sY8yxko6VtMSp4vul/Hxp3DgpOtrflQAAAACAz3QaVq21jZJ+IClX0kZJL1pr1xtj7jfGXNhy2BmSvjTGbJI0VNL8lucjJa0wxmyQ9LSkq1rOh55qnQkYAAAAAEJYRFcOsta+Jc+9p22f+1mb71+W9HIHr6uTZ0ZgOGHXLmnbNsIqAAAAgJDXlWHACBRMrgQAAACgnyCsBpP8fM+WsAoAAAAgxBFWg0lBgZSWJg0Z4u9KAAAAAMCnCKvBhMmVAAAAAPQThNVgUV8vbdggTZzo70oAAAAAwOcIq8Fi40apoYHOKgAAAIB+gbAaLJgJGAAAAEA/QlgNFgUFUkyMdMQR/q4EAAAAAHyOsBos8vOlCROkiAh/VwIAAAAAPkfyCQbWejqrl1zi70oAAACAQ3JVufTZts80InGEspKzlBKbImOM49fZV79PxbuLVbS7SNHh0cpOydaopFGKDI90/Fp9raahxvPeKoqUGpeqKcOnKCKsf8a2/vmug01pqbRrF/erAgAAIOCU7yvXSxte0sJ1C/Xh1g/b7UuKSVJWcpayUrKUnZzt2aZkKys5S2kJaQozhx7oWVFboaKKIhVWFKpo9zfboooibd+7/aDjw0yYRiaO7PBaWSlZGhA5wPH33lO7a3d/854qitq9v9I9pe2OTYxO1DljztH0rOmakTVDI5NG+qnqvkdYDQatkyuxbA0AAEC/9Xnp59q+d7vOzz5f4WHhfq2lqq5Kr3zxihatW6SlxUvVZJs0PnW8HjjzAU3Pmq6yvWXtQuaq7au0eONiNTY3es8RGxGrMcljvOEyNjK2XYDbXbe73TXTE9KVnZKt87LPaxdC6xrrDgp8L254URW1Fe1enxaf5nldy/UmDpuoycMna0jcEJ/8GVlr9XXl11pZulJry9a2q+9QtU3Pmu4N2mOSx2hL5RblFuUqtyhX/9z4T0nS2EFjNSNrhmZkz9C0kdMUFxXnk/oDgbHW+ruGdnJycmxeXp6/ywgsDz0k3XuvVFUlDRzo72oAAAgaDU0N+vfmf+uLnV/o9JGn67hhxx22kwMEosq6St299G499flTsrI6IuUI3XPaPfruhO/26bDX2oZavbHpDS1ct1BvffWW9jft16ikUZpzzBzNPma2JgyZcNghv43NjdpatdUbRtuG2eLdxapvqtfIpJHeINo2kI5JHtPtzmhr97LttVqv17Z7OTJxpCYPn6zJ6Z6vE9JP0MDo7n/mLttbppWlK7Vy20rPtnSldtbslPRN17fde2vp/HblvVlrtXHnRuUWeoLr+1veV11jnaLCo3Ra5mne8NrZzyBQGGM+t9bmdHocYTUIXHGFtHKlVFzs70oAAAh4zbZZH239SAvXLdRLG17yfliUpKFxQ3Vu1rmakTVD07OmO9pRce91a+W2lVq1fZX21O/p9uuNjBJjEpUSm6LkmGSlxKZ4v5Jjk5UYnei3blrrh/B8d77GJI/RhWMvVHxUvF9q6U+stfrH+n/oR+/8SOU15bp16q06MeNEPfLhI1rtXq2RiSN116l36b8m/peiI6J9UkN9U73eLXpXC9ct1Gtfvqa99Xs1LH6YLh9/ueZMmKOpw6c6Eo6abbOabXOf3Zu5t36vVm9frc+2feYNlsW7PZ+1jYzGDh6rKcOneAPsccOOU0xEjPf1VXVV+nz7595g+tm2z+SqdknyBNOjU4/2vHb4ZE0ZPkXHDDlGUeFRjtVf21CrFVtXeMPr+vL1kjwd2ulZ0zXvhHk6ecTJjl3PaYTVUHLUUdK4cdIrr/i7EgAAvCpqK7SgYIH21e/T9KzpmpQ2yW9hylqrVdtXaeG6hfrH+n+opLpEsRGxunDshZpzzBxNHDZR7339nnKLcvVu8bveAHv8sOO9HYmTR5zc5Q+TVXVVyivN837IXbltZbsPqrERsd1+D822WbWNtYfcb2SUFJPkDa+tQTZ1QKrGJI/xdmxGJ49u96G6u6r3V+vz0s+9H8BXlq7U1qqt7Y6JjYjVBWMv0Jxj5uj87PN9FpR8qaGpQVuqtni7fMW7izV4wGBNHj5ZOek5SopJ8mt9xbuLddObNym3KFc56Tl66jtPaVLaJEme3/e3vnpLD3zwgD7d9qnSE9L1k5N/onknzHPkvszq/dVaVrxMb331lhZ/sVgVtRVKjknWrHGzNGfCHE0bOc3vw5B9YWfNTs/f6zYBtGxfmSQpMixSE4ZO0JjkMVpbtlZf7vrS+7qs5Kx2ndlJaZP6fGjutuptWlK0xPtv3OMzHtfc4+b2aQ3dQVgNFfv2SQkJ0n33eb4AAPCzvNI8PbnySS1at0h1jXXe5wfFDmrXtUxPSPd5LRvLN2rhuoVatG6Rvqr4SpFhkTov+zzNOWaOLhh7QYfdv2bbrFXbV3k7Eh+XfKzG5kbFR8XrzFFnesNrdkq2JKmusU757vxvOjDbVnb4QXVK+hRNHj5Zxw87vscfVOub6rW7drcqaiu0u86zbf1qfb6irs33tRVy73W36+QaGe9MrAcOOcxKzlJCdIL32LrGOhW4C7yh+7Ntn+nLnV/KyvP5cEzyGO8H8MnDJ2visIkqcBdo0bpFemnDSyqvKVdidKJmjpup2cfM1lmjz+p1Z2xXzS7vn3NxZbESoxO/6TB30HFOikk65DVbZ1U9cNhp0e4ibancoibb5D02JiKm3e/zESlHfNNZa/m5xkZ2/z8huqu+qV6/+s+vdP8H9ysyLFLzz5qvmybf1GE4tNbq35v/rQc+eEDvb3lfqQNSdftJt+umyTe1+zl3ptk26/PSz733Rn7s+lhNtknxUfG6cOyFmn30bM3InuFoZzAYWGtVUl3Sbmjv5srNOmbIMd6/FznpORo0YJC/S22nqblJTbYpoH9ehNVQ8emn0oknerqqF1/s72oAAJ2w1mpX7a6D7sdqaG44+IN2m8etnbLedMR8qbahVi+uf1G/z/u9Ptv2meIi4zT32Lm6cfKNSotP07vF73o+6BbmejsRE4ZM8Aa/UzNPdey9fV35tRatW6SF6xZqTdkahZkwnTnqTM05Zo4uGXeJUmJTunW+6v3VWr55ufeDeutQwDHJY5QYnai1O9Z6J4VJi09rF0xz0nO6fT2ntf7OtQ1khbu/+b68przd8UPihig7JVv7G/drTdkaNTQ3SJKGxQ9rF0xz0nM0eMDgQ163sblRy4qXaeG6hXrli1dUvb9aQ+KG6LLxl2n2MbN18oiTO70/eG/9Xq3avqpdJ2tz5WZJntCdlpCmPfv3dDqsOjE6sV23eX/j/g5nVU2KSVJ2SvZB90Rmp2RrWPwwVdZVHtQx37ZnmyQp3IRrwtAJ3j+jKcOn6OghRzs6bPWjrR/phjdu0Pry9Zo5bqaeOO8JZQzM6NJrV2xZofkr5iu3KFfJMcm6deqtumXqLUqOTe7w+NI9pd904ore1a7aXZKkSWmTPH9vs2bopBEnBXTgQfAirIaKp5+WbrjBc7/q6NH+rgYAIE8XonRP6UGzT7aGhar9Ve2OzxiYodiIWG+nrNk2H/LcsRGx3g/dyTHJPfqgGBkeqQlDJng/UGcmZvb4nrLi3cX6Y94f9efVf1ZFbYXGDR6nmybfpLnHzlViTOJBx1trtaZsjTf4fbj1Q9U31Ss2IlZnjDrDG17HDhrrrclaq5qGmkN3Elsf1+3W5srNyiv1fE44KeMkzT5mti4/+nINix/Wo/fXkcKKQuUW5mpJ8RLVNtR6w9vk9MkaPnC4Y9fpK9X7q9v/rlYUqXB3ocJNeLt76oYnDO/x70ldY53e/uptLVy3UP/a9C/VNdYpMzFTVxx9hXcYdkNzg9aWrW13j+CG8g3evw+ZiZntgmDbSW4amhpUWVfZ7neho45z6+PwsPAOO8s9+Y+F0j2l7SbMWbltpXeW2tiIWB2fdry35snpk5Wdkt3tP8eK2grdtfQuPbPqGWUmZup35/9OF4y9oNu1StLKbSs1f8V8vfbla0qIStAPpvxAt514mxKiE/Th1g+9IwrW7lgryXMfd+uSKOdmneuzmXGBtgiroeLmm6W//lWqrJSCYGYvAAg1++r3abX7m0k4WpcfaDtcMCIsQqOSRnXYrTnw/sFm26w9+/cc+gP3AR/G2y7z0FU1DTVaX75e9U31kqTUAant7qfqbKmGpuYmvVP4jn6f93u9/dXbCjNhuvioi3Xz5Jt1xqgzuvVBfF/9Pu+9orlFudq0a5MkacTAEUqITvC+39ZaOxIZFvnN/ZlxqTo/+3zNPma2RiWN6nId6Dt79u/R61++roXrFiq3KFeNzY3KGJihHft2eH/OgwcMbvf7ODl9sobGD/Vz5V1jrVXR7qJ23eBV21d57zdOiklSTnqOt/t+uP/ksNZq4bqFui33Nu2q2aXbTrxN951xnyOTV60pW6P5K+brpfUvef8Nqm2sVWRYpE7NPNX7H0fHDj2WGbLR5wiroeKUU6TwcOmDD/xdCYB+ZM/+Pfpj3h/14oYXFRsRe8ihqwdO9DIwemBQf+hpaGrQ2h1rtXLbNxPLrC9f7+38ZAzM0PHDjtcRKUe0W3YgMzGzz2aw7Kr6pvrDdrE6Wqqhvqlez65+Vn/M+6M2V27WsPhhmjdpnuadMM+xjuLXlV8rtzBX//7632q2zYcdGt36+xUXGRcUSzHgYLtqdmnxxsVaUrxEoxJHeX/nRiWNCqmfaWNzozaUb2gXYDsbPl5RW6Eb37xRS4uXasrwKXrqO09p4rCJjtf2xc4v9MQnTygiLEIzsmfojFFnMJMz/I6wGgqam6XEROnaa6Xf/tbf1QDoByrrKvXbT3+rxz99XBW1FTopw3O/UttOX01DzSFfH2bClBSTpLGDxuqOk+/QxUddHLDhtdk2a9OuTe2Cab47X/ub9kuSUmJT2g3tmzx8sqNDTf2hs/sDI8Ii1NDcoGkjp+mmyTfpkqMu6dM1HIFQUttQq4KygkNOzBVuwhUXFaeHz35YN5xwQ0jOrgscCmE10Fx3nZSf373XNDZKa9ZIzzwjXX+9b+oCAEnl+8r1+CeP63crf6fq/dW64MgL9N+n/7emDJ9y0LF1jXXaXbv7kPcX7qrdpaXFS/VVxVcanzpe9552ry4/+nK/dB6bbbO279l+0L16RRVF2rRrk3fSlrjIOE1Km9QumI5OGh1SnZ9DabtUw576Pbr6uKt1zJBj/F0WEJIq6yq9ywJV1FboRyf+qE9mzQYCDWE1kNTXS9HRnvVSs7O799rYWOk3v5GGBff/5gMITNv3bNdj/3lMf/z8j6ptqNWl4y/VPafd0+uhaE3NTXpx/Yuav2K+1pevV3ZKtu459R5ddexVjnfqGpsbtbVq60Gz7xbtLlJRRVG7dSvDTbj33tLslGydkHaCJg+frHGDx9HVAACgjxBWA0lJiTRihPTUU9K8ef6uBkAQa2r2rAnY22C1tWqrHv3oUf1p1Z/U2NyoKydcqbtPvVvjUsc5UaZXs23Wq1+8qgc/eFCr3as1MnGk7jzlTv3X8f/V42VMrLX6cteX3hkt3/v6vXaBNCYiRlnJWZ77SZO/ua80KzlLmYmZDGsFAMDPCKuBJC9PmjxZeu016cIL/V0NgA7UNdZpz/49GjxgcEAO/Wyd8OhXH/9KFbUVGp08usNlGUYnjVZ0RPQhz1NYUahHPnxEzxc8LyOja467RnedepeyUrJ8Wr+1Vm8Xvq0HPnhAn5R8orT4NP3k5J9o3gnzFBcV1+nrK+sqtax4mXdG2a1VWyVJRw46UtPHTNfxacd7/yzSEtIC9j5ZAABAWA0sb74pfec70qefSlMOvv8LQMdqGmrkqnJ1+3VWVvvq9x20NIj3vsq6ioOea+3MHTf0ON00+SZdOeHKgJgt8cAJj84dc65OSDvBO9S1sKLQe9+l5JkkZ0TiiIOC7KABg/Tn1X/W39f+XZFhkfp/k/6ffnLKT5SZmNmn78daq+VfL9cDHzyg975+T6kDUvXjk36smybf5F1PUfJ0kPNK87zh9NOST9VkmzQweqDOHn22d8kFli4BACD4EFYDyZ//7Jkg6euvpZEj/V0NELCstVq3Y503oKzYssI7M6sT2i7B4l1uJeabx+EmXAvXLVRBWYEGRg/Utcddqxsn36ijBh/lWA1dtbNmpx7/5HH99rPfeic8uve0ezU1Y2q746y12lmzs8MJhAorClVeU+49dkDkAN2Yc6NuP+l2pSWk9fVbOsiHWz/U/BXz9U7hO0qKSdKtU29VZmKmcotytbR4qSpqK2RklJOe4w2nU4dPZRgvAABBjrAaSB56SLr3Xqm2Vorp2T1aQKjaVbNL7xa/q9yiXC0pWqLSPaWSpKNTj9aMrBmalDapR0M646Li2q3XmByb3KV7JK21+rjkYz258km9tP4lNTQ36KzRZ+nmyTfrwrEX+nxG2+17tutXH/9Kf8j7gyMTHlXvr1ZRRZFc1S6dPOJkDR4w2OGKey+vNE/zV8zXq1+8KklKT0jX9KzpmpE1Q+eMOScgawYAAD1HWA0kt9wiLVggVVb6uxLA7xqaGvTptk+9k+PklebJyio5JlnnZp2rGVkzND1rujIGZvi7VO3Yt0N/WvUn/THvj3JVuzQ8YbhuOOEGXT/pesc7k1urtuqXH/1Sz6x6Rg3NDd4Jj8anjnf0OoFs065Nqm+q19GpRwfkfcMAAMAZhNVAcvnlnvVSv/jC35UggO2t36sPtnygtPg0ZaVktbt/L1hZa7Vj3w4V7S7SmrI1WlK0RMs2L1P1/mqFmTCdmHGiZ3hn1gzlpOcE7NIhjc2NenPTm/p93u+1pGiJIsIiNHPcTN08+Wadlnlar4JVUUWRd8IjSbrmuGt056l3Kjulm8tcAQAABAnCaiCZNk0yRnrvPX9XggD10daPNPeVudpcudn7XOqAVO8MrwcuvxFIM9Y2NTdp255t39wvecA6l3vr93qPzUzM9IbTs8ecraSYJD9W3jNf7fpKf8j7g57Lf06VdZU6Zsgxuua4a5Qck9yt81hZvb/lfb9PeAQAANDXCKuBZOxYaeJE6R//8HclCDD1TfW6b/l9evQ/j2pk4kj9esav1djceFDoK6kukdU3f1cTohI6DLLZKdlKT0h3fNmO+qZ6fV359UGBtGh3kYp3F6u+qd57bGRYpEYnj26/pEpylsYOHqus5KyACdm9VdNQo4VrF+rJlU9qtXt1j84RaBMeAQAA9AXCaiBJTJSuvVZ64gl/V4IAsm7HOl21+CoVlBXo+uOv169n/FoJ0QkdHlvXWKfNuzd3OOPr5srNamxu9B4bHR6trJSsDtfgHJk48pCET8RQAAAgAElEQVQzqe6r36fi3cXtQnLr91urtqrZNnuPjYuMa9fpbbvNGJgRsMN5fcFaK/ded7ufQVclxyYHxPI4AAAAfamrYdW301rCMwNwdbU0bJi/K0GAaLbNevyTx3X3sruVFJOk12a/pgvHXnjY18RExGhc6jiNSx130L7G5kZtrdqqooqidsNvCysKtbR4qXf9UEkKN+EamTTSGy5rG2u94Xf73u3tzpsSm6LslGydlHGS5h47t13oHRo3NGQ6pL1ljKErCgAA4AOEVV8rK/Nshw71bx0ICFsqt+ja167Ve1+/p4vGXqSnL3haQ+KG9OqcEWERGpM8RmOSx+hcndtun7VW2/du7zDIrty2UrGRscpOydZ52ecd1IUNxvtJAQAAEDoIq77mdnu2dFaDjnuvW09//rSiw6M1I3uGjh16bI/vBbXW6oU1L+iHb/9Q1lo9e+GzunbitT7vThpjlJ6QrvSEdJ028jSfXgsAAABwEmHV11o7q4TVoOGqcumX//Gsd7m/cb+srO5adpeGxg3V9Kzp3q+udkR31uzUDW/coMUbF+u0zNP0/MXPa3TyaB+/CwAAACC4EVZ9rbWzyjDggFe8u1j/++H/6rn852RldfWxV+uuU+9SfFS8lhQtUW5Rrt4ufFsvrHlBknT8sOM9y7Bkz9DJI05WVHjUQed8c9Obuu7167S7brcePedR/fikH/eryYcAAACAnmI2YF/7xS+kn/9cqq+XIjuehRX+9cXOL/Twhw/rb2v+pvCwcF13/HW685Q7NTJp5EHHNttmrdq+SrmFucotytXHJR+rsblR8VHxOnPUmd7wOix+mG7PvV1Pr3paE4ZM0F9n/lXHDj3WD+8OAAAACCyOzgZsjDlP0hOSwiX9yVr7yAH7R0p6VlKqpApJV1lrS1r2PSrp25LCJL0r6VYbaAnZl8rKpEGDCKoBaE3ZGj204iG9uP5FxUTE6Japt+iOk+9QekL6IV8TZsKUk56jnPQc3Xv6vareX63lm5crt8gTXv+16V+SpNiIWNU11uknJ/9ED5z5gKIjovvqbQEAAAAhodOwaowJl/SkpHMllUhaaYx53Vq7oc1hj0laYK193hhzlqSHJc01xpws6RRJrS2lDyVNk/Sec28hwLnd3K8aYPJK8/TgBw/qtS9fU3xUvO485U7ddtJtPZqVd2D0QF101EW66KiLJEmFFYXKLczV59s/17UTr9XpI093unwAAACgX+hKZ3WKpEJrbbEkGWMWSbpIUtuwOl7SbS3fL5f0asv3VlKMpChJRlKkpLLelx1ECKsB46OtH+nBFQ/qncJ3lBSTpJ9P+7l+OPWHSolNcewa2SnZyp6S7dj5AAAAgP6qK2F1uCRXm8clkqYecEyBpFnyDBW+RFKCMWaQtfZjY8xySdvlCau/s9Zu7H3ZQaSsTDrxRH9X0a8V7y7WD976gd4ufFuDBwzWw2c/rJsm36SB0QP9XRoAAACAQ+hKWO1oIcgD7zm9Q9LvjDHXSvpA0jZJjcaYbEnjJGW0HPeuMeZ0a+0H7S5gzDxJ8yQpMzOz69UHAzqrftPQ1KBfffwr/eL9XygiLEK/PPeXujHnRsVFxfm7NAAAAACd6EpYLZE0os3jDEmlbQ+w1pZKmilJxph4SbOstVUtIfQTa+3eln1vSzpRnkDb9vVPS3pa8swG3LO3EoD27pVqagirfvDR1o90wxs3aH35el1y1CX6zfm/UcbAjM5fCAAAACAghHXhmJWSjjDGjDbGREmaLen1tgcYYwYbY1rPdbc8MwNL0lZJ04wxEcaYSHkmV+o/w4BZY7XP7a7drXn/mqdTnztV1fur9drs17T4isUEVQAAACDIdNpZtdY2GmN+IClXnqVrnrXWrjfG3C8pz1r7uqQzJD1sjLHydE1vbnn5y5LOkrRWnqHD71hr/+X82whQrWGVzqrPWWu1cN1C3ZZ7m3bV7NKPT/yxfnHmLxQfFe/v0gAAAAD0QJfWWbXWviXprQOe+1mb71+WJ5ge+LomSTf0ssbgVdYy8TFh1acKKwp145s3amnxUk0ZPkW5V+Vq4rCJ/i4LAAAAQC90KayihxgGfEg1DTXKd+dr8IDBGpU0SlHhUd0+R31TvX750S/1wAcPKCo8Sr87/3f6fs73FR4W7oOKAQAAAPQlwqovud1SWJg0eLC/K/E7a63Wl69XbmGucoty9cGWD7S/ab8kKcyEacTAEcpOyVZWcpZnm5LlfdzR7L0rtqzQDW/coI07N+qy8Zfp8fMeV3pCel+/LQAAAAA+Qlj1pbIyKTVVCu+fnb5dNbv0bvG7yi3K1ZKiJSrd45lE+ujUo3XT5Js0beQ0Ve+vVtHuIhVWFKqwolCLv1isnTU7251nWPwwZSVneQJscraKK4v1l/y/aGTiSL0x5w19+8hv++PtAQAAAPAhwqov9bM1VhubG/VJySfe7mleaZ6srJJjknVu1rmakTVD07Omdzozb1VdlTfAFlW0bHcX6d+b/60FBQsUbsL105N/qp9N+xlrpgIAAAAhirDqS/0krL6y8RW9sOYFLdu8TNX7qxVmwnRixon6+Rk/14ysGcpJz+nWfaSJMYmalDZJk9ImHbSvtqFWdY11So5NdvItAAAAAAgwhFVfKiuTxo3zdxU+9eamNzXzxZnKGJihy8dfrhnZM3T26LN9FiZjI2MVGxnrk3MDAAAACByEVV+xNuQ7q1sqt2juK3M1cdhEfXzdx4qJiPF3SQAAAABCRJi/CwhZlZVSfX3IhtX6pnpd8fIVarJNeumylwiqAAAAABxFZ9VXyso82xBdY/Wn7/5Un277VC9f9rKyU7L9XQ4AAACAEENn1Vfcbs82BDurL294WU98+oRunXqrZo2f5e9yAAAAAIQgwqqvtHZWQyysFlYU6nuvfU9Th0/Vo+c+6u9yAAAAAIQowqqvtHZWQ2gYcG1DrS576TJFhkfqxcteVFR4lL9LAgAAABCiuGfVV9xuKTJSSg6d9UBvfedW5bvz9eaVbyozMdPf5QAAAAAIYXRWfaWszNNVDQuNP+IXCl7QM6ue0d2n3q1vHfEtf5cDAAAAIMSFRpIKRG53yAwBXr9jvb7/5vc1beQ03X/m/f4uBwAAAEA/QFj1Fbc7JCZX2lu/V5e9dJnio+K1cNZCRYQxchwAAACA7xFWfaV1GHAQs9bq+298X1/s/EILZy1UWkKav0sCAAAA0E/QJvOF5mZPWA3yzuozq57R39b+Tfefcb/OGn2Wv8sBAAAA0I/QWfWFXbukpqagDqurt6/WLW/fohlZM3Tv6ff6uxwAAAAA/Qxh1RfKyjzbIB0GXFVXpcteukyDBwzWC5e8oDDDrwkAAACAvsUwYF9wuz3bIOysWmv1vde/py1VW/T+te8rNS7V3yUBAAAA6IcIq74QxGH1iU+f0OKNi/XYuY/p5BEn+7scAAAAAP0U4zt9IUiHAX+w5QP95N2f6KKxF+nHJ/3Y3+UAAAAA6McIq77gdksxMdLAgf6upEtWb1+tS1+8VGf85QxlJmbquYuekzHG32UBAAAA6McYBuwLbrdnCHCAB75PSj7R/BXz9camNzQweqDuPe1e/ejEHyk5NtnfpQEAAADo5wirvlBWFrBDgK21+mDLB3pwxYNaWrxUg2IH6cEzH9TNU25WUkySv8sDAAAAAEmEVd9wu6UxY/xdRTvWWi0pWqIHVzyoD7d+qKFxQ/XYuY/phpwbFB8V7+/yAAAAAKAdwqovlJVJJ53k7yokSc22Wf/68l96cMWDyivNU8bADP32/N/quuOvU2xkrL/LAwAAAIAOEVad1tgolZf7fdmapuYm/XPjPzV/xXytKVujMclj9MwFz+jq465WVHiUX2sDAAAAgM4QVp1WXi5Z69ew+nnp5/ru4u/qy11fauygsVpw8QLNmTBHEWH8uAEAAAAEB9KL0/y8xmpVXZUufelSNTQ16B+X/kOzxs1SeFi4X2oBAAAAgJ4irDrN7fZs/dBZtdbq+29+X64ql1b81wqdNCIw7psFAAAAgO4irDrNj2F1QcECLVq3SA+c+QBBFQAAAEBQC/N3ASHHT8OAv9r1lW5+62ZNGzlNd596d59eGwAAAACcRlh1mtstxcdLcXF9dsn6pnpdufhKRYVH6YVLXuAeVQAAAABBj2HATnO7+3wI8P/8+3+UV5qnf17+T41IHNGn1wYAAAAAX6Cz6rSysj4dAry0eKke/c+jmjdpnmaOm9ln1wUAAAAAXyKsOq0PO6vl+8o195W5OmrwUfq/8/6vT64JAAAAAH2BsOo0t7tPOqvWWn3v9e+porZCi2Yt0oDIAT6/JgAAAAD0lS6FVWPMecaYL40xhcaYuzrYP9IYs8wYs8YY854xJqPl+TONMfltvuqMMRc7/SYCxv790u7dfdJZfXLlk3pj0xt69JxHddyw43x+PQAAAADoS52GVWNMuKQnJZ0vabykOcaY8Qcc9pikBdbaYyXdL+lhSbLWLrfWTrTWTpR0lqQaSUscrD+w7Njh2fo4rK4pW6M7ltyhbx3xLd0y9RafXgsAAAAA/KErndUpkgqttcXW2npJiyRddMAx4yUta/l+eQf7JelSSW9ba2t6WmzA64M1VmsbajXnn3OUFJOk5y56TsYYn10LAAAAAPylK2F1uCRXm8clLc+1VSBpVsv3l0hKMMYMOuCY2ZIWdnQBY8w8Y0yeMSavvLy8CyUFKLfbs/VhZ/X2JbdrQ/kGLbhkgYbEDfHZdQAAAADAn7oSVjtq3dkDHt8haZoxZrWkaZK2SWr0nsCYNEkTJOV2dAFr7dPW2hxrbU5qamqXCg9IPg6rr37xqv6Q9wfdftLtmp413SfXAAAAAIBAENGFY0okjWjzOENSadsDrLWlkmZKkjEmXtIsa21Vm0Mul/SKtbahd+UGuNZhwEOc73huq96m616/TpPSJumhsx9y/PwAAAAAEEi60lldKekIY8xoY0yUPMN5X297gDFmsDGm9Vx3S3r2gHPM0SGGAIcUt1tKSpJiYhw9bVNzk+a+Mld1jXVaOGuhosKjHD0/AAAAAASaTsOqtbZR0g/kGcK7UdKL1tr1xpj7jTEXthx2hqQvjTGbJA2VNL/19caYUfJ0Zt93tPJA5Hb7ZAjwox89quVfL9dvz/+tjhx0pOPnBwAAAIBAY6w98PZT/8rJybF5eXn+LqNnTj9dCguT3nvPsVN+WvKpTnn2FM0aP0uLZi1i9l8AAAAAQc0Y87m1Nqez47oyDBhd5XBndW/9Xl25+EoNHzhcT33nKYIqAAAAgH6jKxMsoascDqu/+s+vVLy7WO9f+76SYpIcOy8AAAAABDo6q06pqZH27JGGDnXkdDv27dBjHz+mmeNm6vSRpztyTgAAAAAIFoRVp7QuW+NQZ3X+B/NV01Cj+WfN7/xgAAAAAAgxhFWnuN2erQOd1c27N+sPeX/Q9yZ+T0cNPqrX5wMAAACAYENYdYqDndX73rtP4WHh+vkZP+/1uQAAAAAgGBFWndLaWe1lWF1TtkZ/XfNX3TLlFg0fONyBwgAAAAAg+BBWndIaVlNTe3Wae5bdo8SYRN116l0OFAUAAAAAwYmw6pSyMmnwYCkyssenWLFlhd786k3decqdSo5NdrA4AAAAAAguhFWn9HKNVWut7lx6p9IT0nXL1FscLAwAAAAAgk+EvwsIGWVlvZoJ+F+b/qWPSz7WU995SgMiBzhYGAAAAAAEHzqrTulFZ7WpuUn3LLtHRw46Ut87/nsOFwYAAAAAwYfOqhOs7VVYfWHNC1pfvl4vXfaSIsL4kQAAAAAAnVUn7N0r1db2aBhwXWOdfrb8Z8pJz9GscbN8UBwAAAAABB/aeE7oxRqrv1/5e7mqXXruoudkjHG4MAAAAAAITnRWndAaVrvZWa2qq9L8FfN17phzdfaYs31QGAAAAAAEJ8KqE8rKPNtudlYf+89jqqit0CPnPOKDogAAAAAgeBFWndCDYcDuvW79+pNf64qjr9CktEk+KgwAAAAAghNh1QlutxQWJg0a1OWXPPD+A6pvqteDZz3ow8IAAAAAIDgRVp1QViYNGSKFh3fp8MKKQj296mldf/z1yk7J9nFxAAAAABB8CKtO6OYaq/+z/H8UFR6ln037mQ+LAgAAAIDgRVh1gtvd5ZmAV29frUXrFulHU3+ktIQ0HxcGAAAAAMGJsOqEsrIud1bvXna3UmJT9NNTfurjogAAAAAgeEX4u4CgZ22Xw+ryzcuVW5Srx859TIkxiX1QHAAAAAAEJzqrvVVZKdXXdzoM2Fqru5bdpYyBGbp5ys19VBwAAAAABCc6q73VxTVWF29crM+2faY/X/hnxUTE9EFhAAAAABC86Kz2VmtY7aSzet9792nc4HG6+rir+6AoAAAAAAhuhNXeKivzbA/TWa1pqNH68vW66tirFBFGMxsAAAAAOkNY7a0uDAN2VbkkSZmJmX1REQAAAAAEPcJqb7ndUmSklJx8yENc1Z6wOmLgiL6qCgAAAACCGmG1t8rKPPerGnPIQ1o7qyMSCasAAAAA0BWE1d5yuzudCbi1szo8YXhfVAQAAAAAQY+w2ltud6czAbuqXBoaN1TREdF9VBQAAAAABDfCam+VlXXaWd1avZXJlQAAAACgGwirvdHUJO3Y0fkw4CoX96sCAAAAQDcQVntj1y5PYD3MMGBrrVzVLmYCBgAAAIBuIKz2RlmZZ3uYzmrV/irtrd9LWAUAAACAbiCs9obb7dkeJqyybA0AAAAAdF+Xwqox5jxjzJfGmEJjzF0d7B9pjFlmjFljjHnPGJPRZl+mMWaJMWajMWaDMWaUc+X7WWtYPcww4NZla+isAgAAAEDXdRpWjTHhkp6UdL6k8ZLmGGPGH3DYY5IWWGuPlXS/pIfb7Fsg6ZfW2nGSpkja4UThAaELw4DprAIAAABA93WlszpFUqG1tthaWy9pkaSLDjhmvKRlLd8vb93fEmojrLXvSpK1dq+1tsaRygOB2y3FxEgJCYc8xFXtUrgJV1p8Wh8WBgAAAADBrSthdbgkV5vHJS3PtVUgaVbL95dISjDGDJJ0pKRKY8xiY8xqY8wvWzq1oaF1jVVjDnmIq9ql9IR0hYeFztsGAAAAAF/rSljtKInZAx7fIWmaMWa1pGmStklqlBQh6bSW/ZMljZF07UEXMGaeMSbPGJNXXl7e9er9ze1mjVUAAAAA8IGuhNUSSW3TVoak0rYHWGtLrbUzrbXHS7q35bmqlteubhlC3CjpVUmTDryAtfZpa22OtTYnNTW1h2/FD9zuw06uJIk1VgEAAACgB7oSVldKOsIYM9oYEyVptqTX2x5gjBlsjGk9192Snm3z2mRjTGsCPUvSht6XHSBahwEfgrXW01klrAIAAABAt3QaVls6oj+QlCtpo6QXrbXrjTH3G2MubDnsDElfGmM2SRoqaX7La5vkGQK8zBizVp4hxc84/i78oaFB2rnzsGG1vKZc+5v2KzMxsw8LAwAAAIDgF9GVg6y1b0l664Dnftbm+5clvXyI174r6dhe1BiYysslaw+/xirL1gAAAABAj3RlGDA60pU1VqtbwirDgAEAAACgWwirPeV2e7aHC6t0VgEAAACgRwirPdUaVg83DLjapejwaKUOCKIZjgEAAAAgABBWe6p1GHAnYTVjYIaM6WipWgAAAADAoRBWe8rtluLjpbi4Qx7iqnIxBBgAAAAAeoCw2lNu92HvV5U8nVUmVwIAAACA7iOs9lRZ2WHDalNzk7ZVbyOsAgAAAEAPEFZ7yu0+7P2q7r1uNdkmhgEDAAAAQA8QVnuqk84qa6wCAAAAQM8RVnti/35p927WWAUAAAAAHyGs9kQXlq3ZWrVVkpSZmNkXFQEAAABASCGs9kRrWO1kGHB8VLwSoxP7qCgAAAAACB2E1Z5wuz3bTsLqiIEjZIzpo6IAAAAAIHQQVnuiNaweZhiwq8rF/aoAAAAA0EOE1Z7owj2rrZ1VAAAAAED3EVZ7wu2WkpKk6OgOd9c31atsbxlhFQAAAAB6iLDaE273Ye9X3Va9TVaWYcAAAAAA0EOE1Z4oK+t0ciVJdFYBAAAAoIcIqz3hdnc6uZIkOqsAAAAA0EOE1Z7oZBgwnVUAAAAA6B3Canft2yft3Xv4sFrlUnJMsuKi4vqwMAAAAAAIHYTV7urqsjUMAQYAAACAHiOsdldrWD1MZ3Vr1VZlJmb2UUEAAAAAEHoIq93ldnu2ndyzyv2qAAAAANBzhNXuag2rhxgGXNNQo4raCsIqAAAAAPQCYbW7ysokY6TU1A53s2wNAAAAAPQeYbW73G5p8GApMrLD3SxbAwAAAAC9R1jtLrf78DMB01kFAAAAgF6L8HcBQScnRxo//pC7WzurwxOG91VFAAAAABByCKvd9d//fdjdriqXhsYNVXREdB8VBAAAAAChh2HADnNVuxgCDAAAAAC9RFh1GGusAgAAAEDvEVYd5qoirAIAAABAbxFWHVRVV6U99XsYBgwAAAAAvURYdVDrTMCZiZl+rgQAAAAAghth1UFbq7ZKEsOAAQAAAKCXCKsOclV5OqsMAwYAAACA3iGsOshV7VK4CVdafJq/SwEAAACAoEZYdZCr2qX0hHSFh4X7uxQAAAAACGpdCqvGmPOMMV8aYwqNMXd1sH+kMWaZMWaNMeY9Y0xGm31Nxpj8lq/XnSw+0LiqXAwBBgAAAAAHdBpWjTHhkp6UdL6k8ZLmGGPGH3DYY5IWWGuPlXS/pIfb7Ku11k5s+brQoboDkquaNVYBAAAAwAld6axOkVRorS221tZLWiTpogOOGS9pWcv3yzvYH/KstSqpLiGsAgAAAIADuhJWh0tytXlc0vJcWwWSZrV8f4mkBGPMoJbHMcaYPGPMJ8aYi3tVbQDbWbNTdY11DAMGAAAAAAd0JayaDp6zBzy+Q9I0Y8xqSdMkbZPU2LIv01qbI+lKSY8bY7IOuoAx81oCbV55eXnXqw8gruqWZWvorAIAAABAr3UlrJZIapvAMiSVtj3AWltqrZ1prT1e0r0tz1W17mvZFkt6T9LxB17AWvu0tTbHWpuTmprak/fhd6yxCgAAAADO6UpYXSnpCGPMaGNMlKTZktrN6muMGWyMaT3X3ZKebXk+2RgT3XqMpFMkbXCq+EBCZxUAAAAAnNNpWLXWNkr6gaRcSRslvWitXW+Mud8Y0zq77xmSvjTGbJI0VNL8lufHScozxhTIM/HSI9ba0AyrVS5Fh0crNS44O8MAAAAAEEgiunKQtfYtSW8d8NzP2nz/sqSXO3jdfyRN6GWNQWFr9VZlDMxQmOnS0rUAAAAAgMMgWTnEVeXiflUAAAAAcAhh1SGuahf3qwIAAACAQwirDmhqbtK26m2EVQAAAABwCGHVAe69bjXZJoYBAwAAAIBDCKsOYNkaAAAAAHAWYdUBrqqWsEpnFQAAAAAcQVh1AJ1VAAAAAHAWYdUBriqX4iLjlBST5O9SAAAAACAkEFYd4Kr2rLFqjPF3KQAAAAAQEgirDnBVu5SZmOnvMgAAAAAgZBBWHeCqcnG/KgAAAAA4iLDaS/VN9XLvdRNWAQAAAMBBhNVe2la9TVaWZWsAAAAAwEGE1V5i2RoAAAAAcB5htZdcVS1hlc4qAAAAADiGsNpLdFYBAAAAwHmE1V5yVbmUHJOsuKg4f5cCAAAAACGDsNpLrmoXQ4ABAAAAwGGE1V5yVbPGKgAAAAA4jbDaS64qwioAAAAAOI2w2gs1DTXaVbuLYcAAAAAA4DDCai+UVJdIkjITM/1cCQAAAACEFsJqL3jXWGUYMAAAAAA4irDaC1urtkoSw4ABAAAAwGGE1V5wVXs6q8MThvu5EgAAAAAILYTVXnBVuTQ0bqiiI6L9XQoAAAAAhBTCai+4ql0MAQYAAAAAHyCs9oKrmjVWAQAAAMAXIvxdQDBzVbl0zuhz/F0GAAAAAB9oaGhQSUmJ6urq/F1KUIqJiVFGRoYiIyN79HrCag9V1VVpT/0ehgEDAAAAIaqkpEQJCQkaNWqUjDH+LieoWGu1a9culZSUaPTo0T06B8OAe6h1JmCGAQMAAAChqa6uToMGDSKo9oAxRoMGDepVV5qw2kOuqpawSmcVAAAACFkE1Z7r7Z8dYbWH6KwCAAAAgO8QVnvIVeVSuAlXWkKav0sBAAAAgC576KGHevS666+/Xhs2bHC4mkMjrPaQq9ql9IR0RYQxRxUAAACA4HGosGqtVXNz8yFf96c//Unjx4/3VVkHIWn10NaqrdyvCgAAAPQXP/qRlJ/v7DknTpQef/ywh1x88cVyuVyqq6vTrbfeqnnz5umdd97RPffco6amJg0ePFjLli3T3r179cMf/lB5eXkyxui+++7TrFmzDjrfXXfdpdraWk2cOFFHH3205s+fr/PPP19nnnmmPv74Y7366qt65JFHtHLlStXW1urS/9/e3QZpVZ4HHP9fQRjeFBfY0o6riDMm4ssuxlXXoWWo6RDSOhGBDsnYMdYOfEji4EzSjq+1yphYh7aWkHFmTagJY0XUoo6jtEIB+wGIS+UlClttiD74giibJVrZuHj1w/OICyzuLrI8B/n/Zpg95z5v1/PMNXu49r7PfWbO5I477gBg8uTJzJ8/n8bGRoYPH87cuXN56qmnGDJkCE888QRjxow5ql+PPatHqLSn5POqkiRJkvrVokWL2LBhAy0tLSxYsICdO3cye/ZsHnvsMTZt2sQjjzwCwLx58xgxYgRbtmxh8+bNXH755d2e7+6772bIkCFs3LiRBx98EIDW1lauueYaXnjhBcaOHctdd91FS0sLmzdvZs2aNWzevPmQ87z//vs0NTWxadMmJk2axP3333/UP7s9q0cgM9mxZwfTvjSt2qFIkiRJOhZ66AHtLwsWLGDZsmUAlEolmpubmTRp0v53l44cORKAFStWsGTJkv3H1dTU9PoaY9BPGLUAAAtUSURBVMeOpampaf/60qVLaW5uprOzkzfffJOXXnqJ+vr6A44ZNGgQV1xxBQAXXXQRzz777JF9wE9hsXoE3vm/d9jbuddhwJIkSZL6zerVq1mxYgVr165l6NChTJ48mYaGBlpbWw/ZNzOP+FUxw4YN27+8fft25s+fz/PPP09NTQ3XXnttt+9KHThw4P7rDRgwgM7OziO69qfp1TDgiJgaEa0R8UpE3NjN9rERsTIiNkfE6oioO2j7KRHxekQsPFqBV5OvrZEkSZLU39rb26mpqWHo0KFs27aNdevW0dHRwZo1a9i+fTsAu3fvBmDKlCksXPhJudXW1nbY8w4cOJAPP/yw22179uxh2LBhjBgxgp07d/LMM88cxU/UNz0WqxExAPgx8DXgXOCbEXHwFFDzgZ9nZj1wJ/DDg7bPA9Z89nCLodReKVbtWZUkSZLUT6ZOnUpnZyf19fXcdtttNDU1UVtbS3NzM9OnT6ehoYFZs2YBcOutt9LW1sb5559PQ0MDq1atOux558yZQ319PVdfffUh2xoaGrjwwgs577zzuO6665g4cWK/fb6eRGZ++g4RlwF/l5lfrazfBJCZP+yyz4vAVzNzR5T7gtsz85TKtouAvwaWA42Z+d1Pu15jY2O2tLR8ho/U/xb+YiHXP3M9b33vLcYMP7ozXkmSJEkqhq1btzJ+/Phqh3Fc6+47jIgNmdnY07G9GQZ8GlDqsr6j0tbVJuDjeZGvAk6OiFER8QXgHygXq4cVEXMioiUiWnbt2tWLkKqr1F5i0IBB1A6rrXYokiRJkvS51JsJlrp7Svfg7tjvAwsj4lrgOeB1oBP4NvB0ZpY+7WHfzGwGmqHcs9qLmKqqtKdE3Sl1fCF8848kSZKkYrr00kvp6Og4oG3x4sVccMEFVYqob3pTrO4Auj6cWQe80XWHzHwDmA4QEcOBGZnZXhlC/EcR8W1gODAoIt7LzEMmaTqe+I5VSZIkSUW3fv36aofwmfSma/B54OyIGBcRg4BvAE923SEiRleG/ALcBCwCyMyrM/OMzDyTcu/rz4/3QhXKw4DPGHFGtcOQJEmSpM+tHovVzOwEvgv8O7AVWJqZL0bEnRHx9cpuk4HWiPgfYAxwVz/FW3X7PtrH67993Z5VSZIkSepHvRkGTGY+DTx9UNvfdll+FHi0h3M8ADzQ5wgLZuf7O+n8qNPX1kiSJElSP3KGoD56rf01AHtWJUmSJKkfWaz2Uam9/BYfe1YlSZIknQjOPPNM3nnnnWN+XYvVPirtqRSr9qxKkiRJUr/p1TOr+kSpvcSwgcM4dfCp1Q5FkiRJ0jFyw/Ib2PjWxqN6zgm/P4F7p977qftMmzaNUqnE3r17mTt3LnPmzGH58uXcfPPN7Nu3j9GjR7Ny5Uree+89rr/+elpaWogIbr/9dmbMmHHI+e677z62b9/OPffcA8ADDzzAhg0b+NGPftTttarJYrWPSntKnD7idCKi2qFIkiRJ+pxbtGgRI0eO5IMPPuDiiy/myiuvZPbs2Tz33HOMGzeO3bt3AzBv3jxGjBjBli1bAGhra+v2fDNnzuSyyy7bX6w+/PDD3HLLLd1ea8aMGYwaNeoYfMruWaz20RdHfZHTTj6t2mFIkiRJOoZ66gHtLwsWLGDZsmUAlEolmpubmTRpEuPGjQNg5MiRAKxYsYIlS5bsP66mpqbb89XW1nLWWWexbt06zj77bFpbW5k4cWK313r55ZctVo8nP/jKD6odgiRJkqQTwOrVq1mxYgVr165l6NChTJ48mYaGBlpbWw/ZNzN7Pfpz1qxZLF26lHPOOYerrrqKiOj2Wnv37j3aH6lPnGBJkiRJkgqovb2dmpoahg4dyrZt21i3bh0dHR2sWbOG7du3A+wfBjxlyhQWLly4/9jDDQMGmD59Oo8//jgPPfQQs2bNOuy1qs1iVZIkSZIKaOrUqXR2dlJfX89tt91GU1MTtbW1NDc3M336dBoaGvYXm7feeittbW2cf/75NDQ0sGrVqsOet6amhnPPPZdXX32VSy655LDXqrbIzGrHcIDGxsZsaWmpdhiSJEmSTnBbt25l/Pjx1Q7juNbddxgRGzKzsadj7VmVJEmSJBWOEyxJkiRJ0ufQpZdeSkdHxwFtixcv5oILLqhSRH1jsSpJkiRJn0Pr16+vdgificOAJUmSJOkwijbHz/Hks353FquSJEmS1I3Bgwfz7rvvWrAegczk3XffZfDgwUd8DocBS5IkSVI36urq2LFjB7t27ap2KMelwYMHU1dXd8THW6xKkiRJUjcGDhzIuHHjqh3GCcthwJIkSZKkwrFYlSRJkiQVjsWqJEmSJKlwomgzW0XELuDVasfRg9HAO9UOQoVnnqgn5oh6wzxRT8wR9YZ5op4cyxwZm5m1Pe1UuGL1eBARLZnZWO04VGzmiXpijqg3zBP1xBxRb5gn6kkRc8RhwJIkSZKkwrFYlSRJkiQVjsXqkWmudgA6Lpgn6ok5ot4wT9QTc0S9YZ6oJ4XLEZ9ZlSRJkiQVjj2rkiRJkqTCsVjto4iYGhGtEfFKRNxY7XhUfRGxKCLejohfdmkbGRHPRsTLlZ811YxR1RcRp0fEqojYGhEvRsTcSru5IgAiYnBE/CIiNlVy5I5K+7iIWF/JkYcjYlC1Y1V1RcSAiHghIp6qrJsjOkBE/DoitkTExohoqbR5v9EBIuLUiHg0IrZV/n9yWdHyxGK1DyJiAPBj4GvAucA3I+Lc6kalAngAmHpQ243Aysw8G1hZWdeJrRP4XmaOB5qA71R+f5gr+lgHcHlmNgATgKkR0QT8PfBPlRxpA/6qijGqGOYCW7usmyPqzh9n5oQuryLxfqOD/TOwPDPPARoo/14pVJ5YrPbNJcArmfmrzPwdsAS4ssoxqcoy8zlg90HNVwI/qyz/DJh2TINS4WTmm5n535Xl31K+IZyGuaKKLHuvsjqw8i+By4FHK+3myAkuIuqAPwN+UlkPzBH1jvcb7RcRpwCTgJ8CZObvMvM3FCxPLFb75jSg1GV9R6VNOtiYzHwTykUK8HtVjkcFEhFnAhcC6zFX1EVleOdG4G3gWeB/gd9kZmdlF+87uhf4G+CjyvoozBEdKoH/iIgNETGn0ub9Rl2dBewC/qXyWMFPImIYBcsTi9W+iW7anE5ZUq9FxHDgMeCGzNxT7XhULJm5LzMnAHWUR/OM7263YxuViiIirgDezswNXZu72dUc0cTM/DLlR9e+ExGTqh2QCuck4MvAfZl5IfA+BRwabrHaNzuA07us1wFvVCkWFdvOiPgDgMrPt6scjwogIgZSLlQfzMx/qzSbKzpEZSjWasrPN58aESdVNnnfObFNBL4eEb+m/CjS5ZR7Ws0RHSAz36j8fBtYRvmPX95v1NUOYEdmrq+sP0q5eC1Unlis9s3zwNmVWfcGAd8AnqxyTCqmJ4FvVZa/BTxRxVhUAJXnyn4KbM3Mf+yyyVwRABFRGxGnVpaHAH9C+dnmVcDMym7myAksM2/KzLrMPJPy/0H+MzOvxhxRFxExLCJO/ngZmAL8Eu836iIz3wJKEfGlStNXgJcoWJ5EpiNF+iIi/pTyXzEHAIsy864qh6Qqi4iHgMnAaGAncDvwOLAUOAN4DfjzzDx4EiadQCLiD4H/ArbwybNmN1N+btVcERFRT3kyiwGU/5i8NDPvjIizKPeijQReAP4iMzuqF6mKICImA9/PzCvMEXVVyYdlldWTgH/NzLsiYhTeb9RFREygPFnbIOBXwF9Suf9QkDyxWJUkSZIkFY7DgCVJkiRJhWOxKkmSJEkqHItVSZIkSVLhWKxKkiRJkgrHYlWSJEmSVDgWq5IkSZKkwrFYlSRJkiQVjsWqJEmSJKlw/h8Uhp9W1JCCAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 创建会话\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "#     记录准确率\n",
    "    acc_train,acc_val,acc_test = [],[],[]\n",
    "#   学习率使用不同速度的递减方式\n",
    "    lr = 0.8\n",
    "#   递减速度\n",
    "    lr_decay = 0.99\n",
    "#   label平移\n",
    "    def label_move(inp,e=0.05):\n",
    "        return inp * (1 - e) + e / 10\n",
    "#     每批训练128个数据，训练12000批\n",
    "    for i in range(12000):\n",
    "        x_train,y_train = mnist.train.next_batch(128)\n",
    "\n",
    "        _,loss = sess.run([step,cross_entropy_loss],feed_dict={x:x_train,y:label_move(y_train),learning_rate:lr})\n",
    "        if i == 4000:\n",
    "            lr_decay = 0.97\n",
    "        if (i + 1) % 200 == 0:\n",
    "            lr = lr * lr_decay\n",
    "            train = sess.run(accuracy,feed_dict={x:x_train,y:y_train})\n",
    "            val = sess.run(accuracy,feed_dict={x:mnist.validation.images,y:mnist.validation.labels})\n",
    "            print(i,train,val,lr)\n",
    "#             存储训练准确率\n",
    "            acc_train.append(train)\n",
    "#             存储验证准确率\n",
    "            acc_val.append(val)\n",
    "    \n",
    "    print(sess.run(accuracy,feed_dict={x:mnist.test.images,y:mnist.test.labels}))\n",
    "# 数据可视化\n",
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(16,6))\n",
    "plt.plot(range(len(acc_train)),acc_train,c='r',label='acc_train')\n",
    "plt.plot(range(len(acc_val)),acc_val,c='g',label='acc_val')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 作业"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何添加隐层数量，添加后有什么效果\n",
    "由于tensorflow的构建是事先设置好计算图的，所以如代码所示，直接创建中间隐层节点即可    \n",
    "对于mnist任务，添加一个隐层后，准确率有明显提高，但是添加2个，3个隐层时，模型提升并不明显，甚至效果会变差，深度更深时模型就无法收敛了，这时出现了梯度消失和梯度扩散的现象，浅层的更新变的非常慢。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何添加神经元个数，起到了什么样的效果\n",
    "在创建每层的神经元权重时，可以设置权重的shape，这个shape实际上就可以反应该层的神经元数量      \n",
    "在这个任务里，测试了单隐层，神经元数据测试了64，100,128,256，500,512，感觉对模型的性能没有太大影响"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何在模型中添加L1/L2正则化，正则化起什么作⽤\n",
    "在设置每个层的权重时，就可以为该层的权重创建正则化项，最终将这些正则化项添加进损失函数中进行计算\n",
    "这里添加了正则化项后，模型的效果有明显提升，而且准确率没有不加正则项震荡的厉害，模型更简单，泛化能力更强"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使⽤不同的初始化⽅式对模型有什么影响\n",
    "本例子先使用了相同参数的正太分布初始化，效果比随机初始化好一些，最后使用了MSRA初始化，模型的收敛速度大大提升"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "本例中，由于使用了梯度下降算法，所以学习率的调整是比较重要的环节，这里我也是花了很多时间去调整，最后使用了等比例递减的方式，在2000次迭代后，验证集的准确率达到了98%，而最后训练完后，对测试集的预测准确率达到了98.46%              \n",
    "添加label平滑后，训练1800就达到了98%，最终在测试集上达到了$ \\large{0.9866} $,提高了约0.2个百分点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
